<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
  <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
  <title>dil.lexer.Lexer</title>
  <link href="./css/style.css" rel="stylesheet" type="text/css"/>
  <!-- <link href="./img/icon.png" rel="icon" type="image/png"/> -->
  <script type="text/javascript" src="./js/jquery.js"></script>
  <script type="text/javascript" src="./js/modules.js"></script>
  <script type="text/javascript" src="./js/quicksearch.js"></script>
  <script type="text/javascript" src="./js/navigation.js"></script>
  <!--<script type="text/javascript" src="./js/jquery.treeview.js"></script>-->
  <script type="text/javascript">
    var g_moduleFQN = "dil.lexer.Lexer";
  </script>
  
</head>
<body>
<div id="content">
  <h1><a href="./htmlsrc/dil.lexer.Lexer.html" class="symbol">dil.lexer.Lexer</a></h1>
  
<p class="sec_header">Author:</p>Aziz Köksal
<p class="sec_header">License:</p>GPL3
 <p><b>Maturity:</b> very high.</p>
<dl>
<dt class="decl">class <a class="symbol _class" name="Lexer" href="./htmlsrc/dil.lexer.Lexer.html#L27" kind="class" beg="27" end="2798">Lexer</a>; <a title="Permalink to this symbol" href="#Lexer" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L27">#</a></dt>
<dd class="ddef">
<div class="summary">The Lexer analyzes the characters of a source text and
 produces a doubly-linked list of tokens.</div>
<dl>
<dt class="decl">SourceText <a class="symbol _variable" name="Lexer.srcText" href="./htmlsrc/dil.lexer.Lexer.html#L29" kind="variable" beg="29" end="29">srcText</a>; <a title="Permalink to this symbol" href="#Lexer.srcText" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L29">#</a></dt>
<dd class="ddef">
<div class="summary">The source text.</div></dd>
<dt class="decl">char* <a class="symbol _variable" name="Lexer.p" href="./htmlsrc/dil.lexer.Lexer.html#L30" kind="variable" beg="30" end="30">p</a>; <a title="Permalink to this symbol" href="#Lexer.p" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L30">#</a></dt>
<dd class="ddef">
<div class="summary">Points to the current character in the source text.</div></dd>
<dt class="decl">char* <a class="symbol _variable" name="Lexer.end" href="./htmlsrc/dil.lexer.Lexer.html#L31" kind="variable" beg="31" end="31">end</a>; <a title="Permalink to this symbol" href="#Lexer.end" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L31">#</a></dt>
<dd class="ddef">
<div class="summary">Points one character past the end of the source text.</div></dd>
<dt class="decl">Token* <a class="symbol _variable" name="Lexer.head" href="./htmlsrc/dil.lexer.Lexer.html#L33" kind="variable" beg="33" end="33">head</a>; <a title="Permalink to this symbol" href="#Lexer.head" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L33">#</a></dt>
<dd class="ddef">
<div class="summary">The head of the doubly linked token list.</div></dd>
<dt class="decl">Token* <a class="symbol _variable" name="Lexer.tail" href="./htmlsrc/dil.lexer.Lexer.html#L34" kind="variable" beg="34" end="34">tail</a>; <a title="Permalink to this symbol" href="#Lexer.tail" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L34">#</a></dt>
<dd class="ddef">
<div class="summary">The tail of the linked list. Set in scan().</div></dd>
<dt class="decl">Token* <a class="symbol _variable" name="Lexer.token" href="./htmlsrc/dil.lexer.Lexer.html#L35" kind="variable" beg="35" end="35">token</a>; <a title="Permalink to this symbol" href="#Lexer.token" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L35">#</a></dt>
<dd class="ddef">
<div class="summary">Points to the current token in the token list.</div></dd>
<dt class="decl">Diagnostics <a class="symbol _variable" name="Lexer.diag" href="./htmlsrc/dil.lexer.Lexer.html#L38" kind="variable" beg="38" end="38">diag</a>; <a title="Permalink to this symbol" href="#Lexer.diag" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L38">#</a></dt>
<dd class="ddef"></dd>
<dt class="decl">LexerError[] <a class="symbol _variable" name="Lexer.errors" href="./htmlsrc/dil.lexer.Lexer.html#L39" kind="variable" beg="39" end="39">errors</a>; <a title="Permalink to this symbol" href="#Lexer.errors" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L39">#</a></dt>
<dd class="ddef"></dd>
<dt class="decl">char* <a class="symbol _variable" name="Lexer.lineBegin" href="./htmlsrc/dil.lexer.Lexer.html#L41" kind="variable" beg="41" end="41">lineBegin</a>; <a title="Permalink to this symbol" href="#Lexer.lineBegin" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L41">#</a></dt>
<dd class="ddef">
<div class="summary">Always points to the first character of the current line.</div></dd>
<dt class="decl">uint <a class="symbol _variable" name="Lexer.lineNum" href="./htmlsrc/dil.lexer.Lexer.html#L43" kind="variable" beg="43" end="43">lineNum</a>; <a title="Permalink to this symbol" href="#Lexer.lineNum" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L43">#</a></dt>
<dd class="ddef">
<div class="summary">Current, actual source text line number.</div></dd>
<dt class="decl">uint <a class="symbol _variable" name="Lexer.lineNum_hline" href="./htmlsrc/dil.lexer.Lexer.html#L44" kind="variable" beg="44" end="44">lineNum_hline</a>; <a title="Permalink to this symbol" href="#Lexer.lineNum_hline" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L44">#</a></dt>
<dd class="ddef">
<div class="summary">Line number set by #line.</div></dd>
<dt class="decl">uint <a class="symbol _variable" name="Lexer.inTokenString" href="./htmlsrc/dil.lexer.Lexer.html#L45" kind="variable" beg="45" end="45">inTokenString</a>; <a title="Permalink to this symbol" href="#Lexer.inTokenString" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L45">#</a></dt>
<dd class="ddef">
<div class="summary">&gt; 0 if inside q{ }</div></dd>
<dt class="decl">NewlineData.FilePaths* <a class="symbol _variable" name="Lexer.filePaths" href="./htmlsrc/dil.lexer.Lexer.html#L47" kind="variable" beg="47" end="47">filePaths</a>; <a title="Permalink to this symbol" href="#Lexer.filePaths" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L47">#</a></dt>
<dd class="ddef">
<div class="summary">Holds the original file path and the modified one (by #line.)</div></dd>
<dt class="decl"><a class="symbol _ctor" name="Lexer.this" href="./htmlsrc/dil.lexer.Lexer.html#L53" kind="ctor" beg="53" end="83">this</a><span class="params">(SourceText <em>srcText</em>, Diagnostics <em>diag</em> = null)</span>; <a title="Permalink to this symbol" href="#Lexer.this" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L53">#</a></dt>
<dd class="ddef">
<div class="summary">Constructs a Lexer object.</div>
<p class="sec_header">Params:</p>
<table class="params">
<tr><td><em>srcText</em></td><td>the UTF-8 source code.</td></tr>
<tr><td><em>diag</em></td><td>used for collecting error messages.</td></tr></table></dd>
<dt class="decl"><a class="symbol _dtor" name="Lexer.~this" href="./htmlsrc/dil.lexer.Lexer.html#L86" kind="dtor" beg="86" end="96">~this</a>(); <a title="Permalink to this symbol" href="#Lexer.~this" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L86">#</a></dt>
<dd class="ddef">
<div class="summary">The destructor deletes the doubly-linked token list.</div></dd>
<dt class="decl">char[] <a class="symbol _function" name="Lexer.text" href="./htmlsrc/dil.lexer.Lexer.html#L98" kind="function" beg="98" end="101">text</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.text" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L98">#</a></dt>
<dd class="ddef"></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanShebang" href="./htmlsrc/dil.lexer.Lexer.html#L105" kind="function" beg="105" end="120">scanShebang</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.scanShebang" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L105">#</a></dt>
<dd class="ddef">
<div class="summary">The "shebang" may optionally appear once at the beginning of a file.
 <pre class="bnf">Shebang := "#!" AnyChar* EndOfLine</pre></div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.finalizeSpecialToken" href="./htmlsrc/dil.lexer.Lexer.html#L123" kind="function" beg="123" end="160">finalizeSpecialToken</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.finalizeSpecialToken" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L123">#</a></dt>
<dd class="ddef">
<div class="summary">Sets the value of the special token.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.newFilePath" href="./htmlsrc/dil.lexer.Lexer.html#L163" kind="function" beg="163" end="169">newFilePath</a><span class="params">(char[] <em>newPath</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.newFilePath" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L163">#</a></dt>
<dd class="ddef">
<div class="summary">Sets a new file path.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.setLineBegin" href="./htmlsrc/dil.lexer.Lexer.html#L171" kind="function" beg="171" end="178">setLineBegin</a><span class="params">(char* <em>p</em>)</span>; <span class="attrs">[<span class="prot">private</span>]</span> <a title="Permalink to this symbol" href="#Lexer.setLineBegin" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L171">#</a></dt>
<dd class="ddef"></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanNext" href="./htmlsrc/dil.lexer.Lexer.html#L183" kind="function" beg="183" end="200">scanNext</a><span class="params">(ref Token* <em>t</em>)</span>; <span class="attrs">[<span class="prot">private</span>]</span> <a title="Permalink to this symbol" href="#Lexer.scanNext" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L183">#</a></dt>
<dd class="ddef">
<div class="summary">Scans the next token in the source text.</div>
Creates a new token if t.next is null and appends it to the list.</dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.peek" href="./htmlsrc/dil.lexer.Lexer.html#L203" kind="function" beg="203" end="206">peek</a><span class="params">(ref Token* <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.peek" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L203">#</a></dt>
<dd class="ddef">
<div class="summary">Advance t one token forward.</div></dd>
<dt class="decl">TOK <a class="symbol _function" name="Lexer.nextToken" href="./htmlsrc/dil.lexer.Lexer.html#L209" kind="function" beg="209" end="213">nextToken</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.nextToken" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L209">#</a></dt>
<dd class="ddef">
<div class="summary">Advance to the next token in the source text.</div></dd>
<dt class="decl">bool <a class="symbol _function" name="Lexer.isNewlineEnd" href="./htmlsrc/dil.lexer.Lexer.html#L216" kind="function" beg="216" end="225">isNewlineEnd</a><span class="params">(char* <em>p</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.isNewlineEnd" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L216">#</a></dt>
<dd class="ddef">
<div class="summary">Returns true if p points to the last character of a Newline.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scan" href="./htmlsrc/dil.lexer.Lexer.html#L230" kind="function" beg="230" end="647">scan</a><span class="params">(ref Token <em>t</em>)</span>; <span class="attrs">[<span class="prot">public</span>]</span> <a title="Permalink to this symbol" href="#Lexer.scan" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L230">#</a></dt>
<dd class="ddef">
<div class="summary">The main method which recognizes the characters that make up a token.</div>
Complicated tokens are scanned in separate methods.</dd>
<dt class="decl">template <a class="symbol _template" name="Lexer.toUint" href="./htmlsrc/dil.lexer.Lexer.html#L650" kind="template" beg="650" end="657">toUint</a><span class="tparams">(char[] T)</span>; <a title="Permalink to this symbol" href="#Lexer.toUint" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L650">#</a></dt>
<dd class="ddef">
<div class="summary">Converts a string literal to an integer.</div>
<dl>
<dt class="decl">uint <a class="symbol _variable" name="Lexer.toUint.toUint" href="./htmlsrc/dil.lexer.Lexer.html#L654" kind="variable" beg="654" end="654">toUint</a>; <span class="attrs">[<span class="stc">const</span>]</span> <a title="Permalink to this symbol" href="#Lexer.toUint.toUint" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L654">#</a></dt>
<dd class="ddef"></dd></dl></dd>
<dt class="decl">template <a class="symbol _template" name="Lexer.case_" href="./htmlsrc/dil.lexer.Lexer.html#L668" kind="template" beg="668" end="674">case_</a><span class="tparams">(char[] str, char[] kind, char[] label)</span>; <a title="Permalink to this symbol" href="#Lexer.case_" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L668">#</a></dt>
<dd class="ddef">
<div class="summary">Constructs case statements. E.g.:
 <pre class="d_code">
<span class="lc">// case_!("&lt;", "Less", "Lcommon") -&gt;</span>
<span class="k">case</span> <span class="n">60u</span>:
  <span class="i">t</span>.<span class="i">kind</span> = <span class="i">TOK</span>.<span class="i">Less</span>;
  <span class="k">goto</span> <span class="i">Lcommon</span>;
</pre></div>
<p class="sec_header">FIXME:</p>Can't use this yet due to a <a href="http://d.puremagic.com/issues/show_bug.cgi?id=1534">bug</a> in DMD.
<dl>
<dt class="decl">char[] <a class="symbol _variable" name="Lexer.case_.case_" href="./htmlsrc/dil.lexer.Lexer.html#L670" kind="variable" beg="670" end="673">case_</a>; <span class="attrs">[<span class="stc">const</span>]</span> <a title="Permalink to this symbol" href="#Lexer.case_.case_" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L670">#</a></dt>
<dd class="ddef"></dd></dl></dd>
<dt class="decl">template <a class="symbol _template" name="Lexer.case_L4" href="./htmlsrc/dil.lexer.Lexer.html#L677" kind="template" beg="677" end="680">case_L4</a><span class="tparams">(char[] str, TOK kind)</span>; <a title="Permalink to this symbol" href="#Lexer.case_L4" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L677">#</a></dt>
<dd class="ddef">
<dl>
<dt class="decl">char[] <a class="symbol _variable" name="Lexer.case_L4.case_L4" href="./htmlsrc/dil.lexer.Lexer.html#L679" kind="variable" beg="679" end="679">case_L4</a>; <span class="attrs">[<span class="stc">const</span>]</span> <a title="Permalink to this symbol" href="#Lexer.case_L4.case_L4" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L679">#</a></dt>
<dd class="ddef"></dd></dl></dd>
<dt class="decl">template <a class="symbol _template" name="Lexer.case_L3" href="./htmlsrc/dil.lexer.Lexer.html#L682" kind="template" beg="682" end="685">case_L3</a><span class="tparams">(char[] str, TOK kind)</span>; <a title="Permalink to this symbol" href="#Lexer.case_L3" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L682">#</a></dt>
<dd class="ddef">
<dl>
<dt class="decl">char[] <a class="symbol _variable" name="Lexer.case_L3.case_L3" href="./htmlsrc/dil.lexer.Lexer.html#L684" kind="variable" beg="684" end="684">case_L3</a>; <span class="attrs">[<span class="stc">const</span>]</span> <a title="Permalink to this symbol" href="#Lexer.case_L3.case_L3" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L684">#</a></dt>
<dd class="ddef"></dd></dl></dd>
<dt class="decl">template <a class="symbol _template" name="Lexer.case_L2" href="./htmlsrc/dil.lexer.Lexer.html#L687" kind="template" beg="687" end="690">case_L2</a><span class="tparams">(char[] str, TOK kind)</span>; <a title="Permalink to this symbol" href="#Lexer.case_L2" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L687">#</a></dt>
<dd class="ddef">
<dl>
<dt class="decl">char[] <a class="symbol _variable" name="Lexer.case_L2.case_L2" href="./htmlsrc/dil.lexer.Lexer.html#L689" kind="variable" beg="689" end="689">case_L2</a>; <span class="attrs">[<span class="stc">const</span>]</span> <a title="Permalink to this symbol" href="#Lexer.case_L2.case_L2" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L689">#</a></dt>
<dd class="ddef"></dd></dl></dd>
<dt class="decl">template <a class="symbol _template" name="Lexer.case_L1" href="./htmlsrc/dil.lexer.Lexer.html#L692" kind="template" beg="692" end="695">case_L1</a><span class="tparams">(char[] str, TOK kind)</span>; <a title="Permalink to this symbol" href="#Lexer.case_L1" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L692">#</a></dt>
<dd class="ddef">
<dl>
<dt class="decl">char[] <a class="symbol _variable" name="Lexer.case_L1.case_L3" href="./htmlsrc/dil.lexer.Lexer.html#L694" kind="variable" beg="694" end="694">case_L3</a>; <span class="attrs">[<span class="stc">const</span>]</span> <a title="Permalink to this symbol" href="#Lexer.case_L1.case_L3" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L694">#</a></dt>
<dd class="ddef"></dd></dl></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scan_" href="./htmlsrc/dil.lexer.Lexer.html#L699" kind="function" beg="699" end="1112">scan_</a><span class="params">(ref Token <em>t</em>)</span>; <span class="attrs">[<span class="prot">public</span>]</span> <a title="Permalink to this symbol" href="#Lexer.scan_" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L699">#</a></dt>
<dd class="ddef">
<div class="summary">An alternative scan method.
 Profiling shows it's a bit slower.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanBlockComment" href="./htmlsrc/dil.lexer.Lexer.html#L1117" kind="function" beg="1117" end="1157">scanBlockComment</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanBlockComment" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1117">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a block comment.</div>
<pre class="bnf">BlockComment := "/*" AnyChar* "*/"</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanNestedComment" href="./htmlsrc/dil.lexer.Lexer.html#L1162" kind="function" beg="1162" end="1210">scanNestedComment</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanNestedComment" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1162">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a nested comment.</div>
<pre class="bnf">NestedComment := "/+" (AnyChar* | NestedComment) "+/"</pre></dd>
<dt class="decl">char <a class="symbol _function" name="Lexer.scanPostfix" href="./htmlsrc/dil.lexer.Lexer.html#L1215" kind="function" beg="1215" end="1231">scanPostfix</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.scanPostfix" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1215">#</a></dt>
<dd class="ddef">
<div class="summary">Scans the postfix character of a string literal.</div>
<pre class="bnf">PostfixChar := "c" | "w" | "d"</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanNormalStringLiteral" href="./htmlsrc/dil.lexer.Lexer.html#L1236" kind="function" beg="1236" end="1291">scanNormalStringLiteral</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanNormalStringLiteral" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1236">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a normal string literal.</div>
<pre class="bnf">NormalStringLiteral := '"' AnyChar* '"'</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanCharacterLiteral" href="./htmlsrc/dil.lexer.Lexer.html#L1296" kind="function" beg="1296" end="1325">scanCharacterLiteral</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanCharacterLiteral" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1296">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a character literal.</div>
<pre class="bnf">CharLiteral := "'" AnyChar "'"</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanRawStringLiteral" href="./htmlsrc/dil.lexer.Lexer.html#L1330" kind="function" beg="1330" end="1383">scanRawStringLiteral</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanRawStringLiteral" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1330">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a raw string literal.</div>
<pre class="bnf">RawStringLiteral := 'r"' AnyChar* '"' | "`" AnyChar* "`"</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanHexStringLiteral" href="./htmlsrc/dil.lexer.Lexer.html#L1388" kind="function" beg="1388" end="1468">scanHexStringLiteral</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanHexStringLiteral" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1388">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a hexadecimal string literal.</div>
<pre class="bnf">HexStringLiteral := 'x"' (HexChar HexChar)* '"'</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanDelimitedStringLiteral" href="./htmlsrc/dil.lexer.Lexer.html#L1479" kind="function" beg="1479" end="1479">scanDelimitedStringLiteral</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanDelimitedStringLiteral" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1479">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a delimited string literal.</div>
<pre class="bnf">
DelimitedStringLiteral := 'q"' OpeningDelim AnyChar* MatchingDelim '"'
OpeningDelim  := "[" | "(" | "{" | "<" | Identifier EndOfLine
MatchingDelim := "]" | ")" | "}" | ">" | EndOfLine Identifier
</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanTokenStringLiteral" href="./htmlsrc/dil.lexer.Lexer.html#L1483" kind="function" beg="1483" end="1483">scanTokenStringLiteral</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanTokenStringLiteral" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1483">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a token string literal.</div>
<pre class="bnf">TokenStringLiteral := "q{" Token* "}"</pre></dd>
<dt class="decl">dchar <a class="symbol _function" name="Lexer.scanEscapeSequence" href="./htmlsrc/dil.lexer.Lexer.html#L1779" kind="function" beg="1779" end="1904">scanEscapeSequence</a><span class="params">(ref bool <em>isBinary</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanEscapeSequence" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1779">#</a></dt>
<dd class="ddef">
<div class="summary">Scans an escape sequence.</div>
<pre class="bnf">
EscapeSequence := "\\" (Binary | C)
Binary := Octal{1,3} | ("x" Hex{2}) | ("u" Hex{4}) | ("U" Hex{8})
C := "'" | '"' | "?" | "\\" | "a" | "b" | "f" | "n" | "r" | "t" | "v"
</pre>
<p class="sec_header">Params:</p>
<table class="params">
<tr><td><em>isBinary</em></td><td>set to true for octal and hexadecimal escapes.</td></tr></table>
<p class="sec_header">Returns:</p>the escape value.</dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanNumber" href="./htmlsrc/dil.lexer.Lexer.html#L1917" kind="function" beg="1917" end="2202">scanNumber</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanNumber" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L1917">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a number literal.</div>
<pre class="bnf">
IntegerLiteral := (Dec | Hex | Bin | Oct) Suffix?
Dec := ("0" | [1-9] [0-9_]*)
Hex := "0" [xX] "_"* [0-9a-zA-Z] [0-9a-zA-Z_]*
Bin := "0" [bB] "_"* [01] [01_]*
Oct := "0" [0-7_]*
Suffix := ("L" [uU]? | [uU] "L"?)
</pre>
<p class="sec_header">Invalid:</p>"0b_", "0x_", "._" etc.</dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanReal" href="./htmlsrc/dil.lexer.Lexer.html#L2218" kind="function" beg="2218" end="2265">scanReal</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanReal" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2218">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a floating point number literal.</div>
<pre class="bnf">
FloatLiteral := Float [fFL]? i?
Float        := DecFloat | HexFloat
DecFloat     := ([0-9] [0-9_]* "." [0-9_]* DecExponent?) |
                "." [0-9] [0-9_]* DecExponent? |
                [0-9] [0-9_]* DecExponent
DecExponent  := [eE] [+-]? [0-9] [0-9_]*
HexFloat     := "0" [xX] (HexDigits "." HexDigits |
                          "." [0-9a-zA-Z] HexDigits? |
                          HexDigits) HexExponent
HexExponent := [pP] [+-]? [0-9] [0-9_]*
</pre></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanHexReal" href="./htmlsrc/dil.lexer.Lexer.html#L2268" kind="function" beg="2268" end="2307">scanHexReal</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanHexReal" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2268">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a hexadecimal floating point number literal.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.finalizeFloat" href="./htmlsrc/dil.lexer.Lexer.html#L2313" kind="function" beg="2313" end="2344">finalizeFloat</a><span class="params">(ref Token <em>t</em>, string <em>buffer</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.finalizeFloat" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2313">#</a></dt>
<dd class="ddef">
<div class="summary">Sets the value of the token.</div>
<p class="sec_header">Params:</p>
<table class="params">
<tr><td><em>t</em></td><td>receives the value.</td></tr>
<tr><td><em>buffer</em></td><td>the well-formed float number.</td></tr></table></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanSpecialTokenSequence" href="./htmlsrc/dil.lexer.Lexer.html#L2349" kind="function" beg="2349" end="2458">scanSpecialTokenSequence</a><span class="params">(ref Token <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.scanSpecialTokenSequence" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2349">#</a></dt>
<dd class="ddef">
<div class="summary">Scans a special token sequence.</div>
<pre class="bnf">SpecialTokenSequence := "#line" Integer Filespec? EndOfLine</pre></dd>
<dt class="decl">Token* <a class="symbol _function" name="Lexer.insertEmptyTokenBefore" href="./htmlsrc/dil.lexer.Lexer.html#L2464" kind="function" beg="2464" end="2480">insertEmptyTokenBefore</a><span class="params">(Token* <em>t</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.insertEmptyTokenBefore" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2464">#</a></dt>
<dd class="ddef">
<div class="summary">Inserts an empty dummy token (TOK.Empty) before t.</div>
Useful in the parsing phase for representing a node in the AST
 that doesn't consume an actual token from the source text.</dd>
<dt class="decl">uint <a class="symbol _function" name="Lexer.errorLineNumber" href="./htmlsrc/dil.lexer.Lexer.html#L2483" kind="function" beg="2483" end="2486">errorLineNumber</a><span class="params">(uint <em>lineNum</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.errorLineNumber" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2483">#</a></dt>
<dd class="ddef">
<div class="summary">Returns the error line number.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.error" href="./htmlsrc/dil.lexer.Lexer.html#L2489" kind="function" beg="2489" end="2492">error</a><span class="params">(char* <em>columnPos</em>, char[] <em>msg</em>, ...)</span>; <a title="Permalink to this symbol" href="#Lexer.error" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2489">#</a></dt>
<dt class="decl">void <a class="symbol _function" name="Lexer.error:2" href="./htmlsrc/dil.lexer.Lexer.html#L2495" kind="function" beg="2495" end="2498">error</a><span class="params">(char* <em>columnPos</em>, MID <em>mid</em>, ...)</span>; <a title="Permalink to this symbol" href="#Lexer.error:2" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2495">#</a></dt>
<dt class="decl">void <a class="symbol _function" name="Lexer.error:3" href="./htmlsrc/dil.lexer.Lexer.html#L2501" kind="function" beg="2501" end="2504">error</a><span class="params">(uint <em>lineNum</em>, char* <em>lineBegin</em>, char* <em>columnPos</em>, MID <em>mid</em>, ...)</span>; <a title="Permalink to this symbol" href="#Lexer.error:3" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2501">#</a></dt>
<dd class="ddef">
<div class="summary">Forwards error parameters.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.error_" href="./htmlsrc/dil.lexer.Lexer.html#L2512" kind="function" beg="2512" end="2523">error_</a><span class="params">(uint <em>lineNum</em>, char* <em>lineBegin</em>, char* <em>columnPos</em>, char[] <em>msg</em>, TypeInfo[] <em>_arguments</em>, va_list <em>_argptr</em>)</span>; <a title="Permalink to this symbol" href="#Lexer.error_" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2512">#</a></dt>
<dd class="ddef">
<div class="summary">Creates an error report and appends it to a list.</div>
<p class="sec_header">Params:</p>
<table class="params">
<tr><td><em>lineNum</em></td><td>the line number.</td></tr>
<tr><td><em>lineBegin</em></td><td>points to the first character of the current line.</td></tr>
<tr><td><em>columnPos</em></td><td>points to the character where the error is located.</td></tr>
<tr><td><em>msg</em></td><td>the message.</td></tr></table></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.scanAll" href="./htmlsrc/dil.lexer.Lexer.html#L2526" kind="function" beg="2526" end="2530">scanAll</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.scanAll" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2526">#</a></dt>
<dd class="ddef">
<div class="summary">Scans the whole source text until EOF is encountered.</div></dd>
<dt class="decl">Token* <a class="symbol _function" name="Lexer.firstToken" href="./htmlsrc/dil.lexer.Lexer.html#L2535" kind="function" beg="2535" end="2538">firstToken</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.firstToken" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2535">#</a></dt>
<dd class="ddef">
<div class="summary">Returns the first token of the source text.
 This can be the EOF token.</div>
<p class="sec_header">Structure:</p>HEAD -&gt; Newline -&gt; First Token</dd>
<dt class="decl">bool <a class="symbol _function" name="Lexer.isIdentifierString" href="./htmlsrc/dil.lexer.Lexer.html#L2541" kind="function" beg="2541" end="2553">isIdentifierString</a><span class="params">(char[] <em>str</em>)</span>; <span class="attrs">[<span class="stc">static</span>]</span> <a title="Permalink to this symbol" href="#Lexer.isIdentifierString" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2541">#</a></dt>
<dd class="ddef">
<div class="summary">Returns true if str is a valid D identifier.</div></dd>
<dt class="decl">bool <a class="symbol _function" name="Lexer.isReservedIdentifier" href="./htmlsrc/dil.lexer.Lexer.html#L2557" kind="function" beg="2557" end="2565">isReservedIdentifier</a><span class="params">(char[] <em>str</em>)</span>; <span class="attrs">[<span class="stc">static</span>]</span> <a title="Permalink to this symbol" href="#Lexer.isReservedIdentifier" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2557">#</a></dt>
<dd class="ddef">
<div class="summary">Returns true if str is a keyword or
 a special token (__FILE__, __LINE__ etc.)</div></dd>
<dt class="decl">bool <a class="symbol _function" name="Lexer.isValidUnreservedIdentifier" href="./htmlsrc/dil.lexer.Lexer.html#L2568" kind="function" beg="2568" end="2571">isValidUnreservedIdentifier</a><span class="params">(char[] <em>str</em>)</span>; <span class="attrs">[<span class="stc">static</span>]</span> <a title="Permalink to this symbol" href="#Lexer.isValidUnreservedIdentifier" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2568">#</a></dt>
<dd class="ddef">
<div class="summary">Returns true if this is a valid identifier and if it's not reserved.</div></dd>
<dt class="decl">bool <a class="symbol _function" name="Lexer.isUnicodeAlpha" href="./htmlsrc/dil.lexer.Lexer.html#L2577" kind="function" beg="2577" end="2627">isUnicodeAlpha</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.isUnicodeAlpha" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2577">#</a></dt>
<dd class="ddef">
<div class="summary">Returns true if the current character to be decoded is
 a Unicode alpha character.</div>
The current pointer 'p' is set to the last trailbyte if true is returned.</dd>
<dt class="decl">dchar <a class="symbol _function" name="Lexer.decodeUTF8" href="./htmlsrc/dil.lexer.Lexer.html#L2630" kind="function" beg="2630" end="2711">decodeUTF8</a><span class="params">()</span>; <a title="Permalink to this symbol" href="#Lexer.decodeUTF8" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2630">#</a></dt>
<dd class="ddef">
<div class="summary">Decodes the next UTF-8 sequence.</div></dd>
<dt class="decl">void <a class="symbol _function" name="Lexer.encodeUTF8" href="./htmlsrc/dil.lexer.Lexer.html#L2714" kind="function" beg="2714" end="2764">encodeUTF8</a><span class="params">(ref char[] <em>str</em>, dchar <em>d</em>)</span>; <span class="attrs">[<span class="stc">static</span>]</span> <a title="Permalink to this symbol" href="#Lexer.encodeUTF8" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2714">#</a></dt>
<dd class="ddef">
<div class="summary">Encodes the character d and appends it to str.</div></dd>
<dt class="decl">char[] <a class="symbol _function" name="Lexer.formatBytes" href="./htmlsrc/dil.lexer.Lexer.html#L2768" kind="function" beg="2768" end="2777">formatBytes</a><span class="params">(char* <em>start</em>, char* <em>end</em>)</span>; <span class="attrs">[<span class="stc">static</span>]</span> <a title="Permalink to this symbol" href="#Lexer.formatBytes" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2768">#</a></dt>
<dd class="ddef">
<div class="summary">Formats the bytes between start and end (excluding end.)</div>
<p class="sec_header">Returns:</p>e.g.: "abc" -&gt; "\x61\x62\x63"</dd>
<dt class="decl">string <a class="symbol _function" name="Lexer.findInvalidUTF8Sequence" href="./htmlsrc/dil.lexer.Lexer.html#L2781" kind="function" beg="2781" end="2797">findInvalidUTF8Sequence</a><span class="params">(string <em>str</em>)</span>; <span class="attrs">[<span class="stc">static</span>]</span> <a title="Permalink to this symbol" href="#Lexer.findInvalidUTF8Sequence" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2781">#</a></dt>
<dd class="ddef">
<div class="summary">Searches for an invalid UTF-8 sequence in str.</div>
<p class="sec_header">Returns:</p>a formatted string of the invalid sequence (e.g. "\xC0\x80").</dd></dl></dd>
<dt class="decl"><a class="symbol _unittest" name="unittest" href="./htmlsrc/dil.lexer.Lexer.html#L2801" kind="unittest" beg="2801" end="2873">unittest</a>; <a title="Permalink to this symbol" href="#unittest" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2801">#</a></dt>
<dd class="ddef">
<div class="summary">Tests the lexer with a list of tokens.</div></dd>
<dt class="decl"><a class="symbol _unittest" name="unittest:2" href="./htmlsrc/dil.lexer.Lexer.html#L2876" kind="unittest" beg="2876" end="2900">unittest</a>; <a title="Permalink to this symbol" href="#unittest:2" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2876">#</a></dt>
<dd class="ddef">
<div class="summary">Tests the Lexer's peek() method.</div></dd>
<dt class="decl"><a class="symbol _unittest" name="unittest:3" href="./htmlsrc/dil.lexer.Lexer.html#L2902" kind="unittest" beg="2902" end="2910">unittest</a>; <a title="Permalink to this symbol" href="#unittest:3" class="symlink">¶</a><a title="Go to the HTML source file" class="srclink" href="./htmlsrc/dil.lexer.Lexer.html#L2902">#</a></dt>
<dd class="ddef"></dd></dl>
</div>
<div id="footer">
  <p>Copyright © 2007-2008, Aziz Köksal. All rights reserved.</p>
  <p>Page generated by <a href="http://code.google.com/p/dil">dil</a> on Fri Dec 26 04:12:56 2008. Rendered by <a href="http://code.google.com/p/dil/wiki/Kandil">kandil</a>.</p>
</div>
</body>
</html>